{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "#encoding:UTF-8\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import time\n",
    "from datetime import timedelta\n",
    "import math\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST-data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST-data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST-data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST-data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "data = input_data.read_data_sets('MNIST-data/', one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "训练数据共：55000\n",
      "测试数据共：10000\n",
      "第一个测试图的维度：(784,)\n",
      "第一个测试图的真实值：[ 0.  0.  0.  0.  0.  0.  0.  1.  0.  0.]\n"
     ]
    }
   ],
   "source": [
    "print \"训练数据共：\"+str(len(data.train.images))\n",
    "print \"测试数据共：\"+str(len(data.test.images))\n",
    "print \"第一个测试图的维度：\"+str(data.test.images[0].shape)\n",
    "print \"第一个测试图的真实值：\"+str(data.test.labels[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_images(images, cls_true, cls_pred=None):\n",
    "    assert len(images) == len(cls_true) == 9\n",
    "    \n",
    "    # Create figure with 3x3 sub-plots.\n",
    "    fig, axes = plt.subplots(3, 3)\n",
    "    fig.subplots_adjust(hspace=0.3, wspace=0.3)\n",
    "\n",
    "    for i, ax in enumerate(axes.flat):\n",
    "        # Plot image.\n",
    "        ax.imshow(images[i].reshape((28,28)), cmap='binary')\n",
    "\n",
    "        # Show true and predicted classes.\n",
    "        if cls_pred is None:\n",
    "            xlabel = \"label: {0}\".format(cls_true[i])\n",
    "        else:\n",
    "            xlabel = \"True: {0}, Pred: {1}\".format(cls_true[i], cls_pred[i])\n",
    "\n",
    "        ax.set_xlabel(xlabel)\n",
    "        \n",
    "        # Remove ticks from the plot.\n",
    "        ax.set_xticks([])\n",
    "        ax.set_yticks([])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUMAAAD5CAYAAAC9FVegAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHq5JREFUeJzt3XmUFNXd//H3F8QgoiyBKCIwT0QRJKIyPgajiBsBRBCC\nihLgGKMRDG4RjQRxJ4IbPxE3PKi/gEFFQCQCKihCgACyyK4ooMQFBzy4I8t9/pi5Xd0zPcz0THdV\nT/t5ncOZmqrqqtt96Tvfuqs55xAR+amrFnUCRESygQpDERFUGIqIACoMRUQAFYYiIoAKQxERQIWh\niAigwlBEBFBhKCICwAGpnNygQQOXl5eXoaRkn82bN1NQUGBRpyNMyuPcpzxOLqXCMC8vj6VLl1Y8\nVVVMfn5+1EkInfI49ymPk9NjsogIKgxFRAAVhiIigApDERFAhaGICJBia7JIRd1///0AfP/99wC8\n++67AEyaNKnEuQMGDACgXbt2APTt2zeMJMpPnCJDEREUGUqGXXzxxQC8+OKLSY+blewL+/jjjwPw\nxhtvAHDGGWcA0LRp00wkUSL03nvvAdCiRQsAHn74YQAGDRoUeloUGYqIoMhQMsBHg1B6RHjssccC\n0KlTJwA+/PDD2LFp06YBsHHjRgDGjx8PwJAhQ9KfWInU8uXLAahWrTAua9y4cWRpUWQoIoIiQ0kj\nP951ypQpJY61bt0aCKK+Bg0aAFC7dm0Afvzxx9i5p5xyCgArV64EYPv27RlKsURtxYoVQPD/oGfP\nnpGlRZGhiAghRIa+H9nYsWMBOOKII2LHatasCUCfPn0AOPzwwwFo3rx5ppMlGfDpp58C4JyL7fMR\n4axZswBo1KhR0tf6fogA69atSzjWtWvXtKZTordq1SoARo8eDUC/fv2iTA6gyFBEBAghMhw8eDBQ\nOMFiaXy/skMPPRSAVq1apeXeTZo0AeCmm24Cfppz14Xp/PPPB4JWYIBDDjkEgPr16+/3tc8//3xs\nO77+UHLThg0bAPj222+BxB4IUVFkKCKCCkMRESCEx+SnnnoKCLpJxD8Cr127Fgg6Xr711lsALFq0\nCAiGX3300UelXr9GjRpA0FXDV+LHX8c/LusxORzNmjUr97n33XcfEAzLiue72PifkjtGjhwJFC5B\nANnx3VRkKCJCCJHh2WefnfAznh+K5X355ZdAECn6vxZLliwp9fo/+9nPgGCgtx/mBbBjxw4Ajjrq\nqAqlXTJn+vTpAAwbNgyAXbt2xY4ddthhANx7770A1KpVK+TUSSbEN6L677T/3h588MFRJCmBIkMR\nEbJsOF69evUAOOussxL2J4sqi3vppZeAILoEOP744wHo3bt3upIoaeKH7sVHhJ7vZuGn7pLcMHfu\n3BL7GjZsGEFKklNkKCJClkWGFbFt2zYABg4cCCQOBfP1UWV1+JXwXHDBBUAwPM/r379/bPvuu+8O\nNU0SDr/UQzw/ICIbKDIUESEHIsMxY8YAQYRYt27d2DHfUiXR8/0/FyxYAAR1hb7OaOjQobFz/XRO\nkhsWLlwIwNNPPx3bd+KJJwJw7rnnRpKmZBQZiohQhSPD+fPnA0FfNO/ll1+ObfvpoyR6ftLOgoKC\nhP1++jb1Bc1ds2fPBhJ7evg+xn4av2ygyFBEBBWGIiJAFX5MfvXVV4Fg7rtzzjkHgHbt2kWWJinJ\nr3nih1h6HTp0AODOO+8MO0kSMj9JS7wLL7wwgpTsnyJDERGqYGT4/fffAzBz5kwgmKjhjjvuAIIp\nvSQ68avZDR8+HCg5e/UJJ5wAqBtNLvvss88AmDdvHpA4iUqPHj0iSdP+KDIUEaEKRoZ+MlBfB9W5\nc2cATj311MjSJIkeeOCB2PbixYsTjvnheKorzH3PPPMMAJ9//jkQfFezlSJDERGqSGToJwIFuOuu\nuwCoU6cOALfeemskaZLSPfjgg6Ue88MnVVeY+7Zs2ZLwu5+iL1spMhQRIcsjQ98qec0118T27dmz\nB4AuXboA6ldY1fg8LU+rv4/+/bm7d+8GYOfOnSXO9UO9HnrooaTXql69emx7xIgRgJYTyLRXXnkl\n4feuXbtGlJLyUWQoIoIKQxERIEsfk/fu3QsEM1ts2rQpdqx58+ZA0JAiVYtfl6Y8LrroIgAaNWoE\nBF00Jk6cWKk0+NX34udQlPTxnax9flUVigxFRMjSyPCDDz4AghXU4vluG5r/Lnv5xi2AqVOnVvg6\nL7zwQpnn+MaVatUS/65369YNCNbejnfaaadVOE1StilTpgBBY6ef1TrbVztUZCgiQpZFhr6TZseO\nHRP233///bHtbG+eF5g8eXJse+TIkUDJiRq8tWvXAvuvB7z88ssBaNasWYljv/vd7wBo2bJlxRIr\nafPdd98BMGPGjIT9frqu+O5N2UiRoYgIWRYZPvHEE0DJYTzxdQ1mFmqapHLKuy7uc889l+GUSKb5\n+lu/QmX37t0BuPbaayNLUyoUGYqIkCWRoe+X9Mgjj0ScEhGpKB8Z+nWSqxpFhiIiZElk6NdA/vrr\nrxP2+9Emmu5JRDJNkaGICCoMRUSALHlMLs6vnDZ79mwA6tevH2VyROQnQJGhiAhZEhnecsstCT9F\nRMKmyFBEBDDnXPlPNvsC2FLmibmjmXOuYdSJCJPyOPcpj5NLqTAUEclVekwWEUGFoYgIkOHC0My+\nKeN4npmtTvGaz5hZrzLOGWxmK4r+rTazvWamzooZEGEe9zGzd81slZktMLM2qdxDyi/CPD7WzBaa\n2S4zuzGV61dEVnStSTfn3H3AfQBmdj5wvXNuR7SpkjTbBJzhnPvSzDoDTwKnRJwmSa8dwDXABWHc\nLJTHZDOrbWazzWxZ0V/y7nGHDzCzCWa2zswmmVmtote0NbO5ZvaOmc0ys0YVvP0lwD8r/SZkv8LO\nY+fcAufcl0W/LgKOTOPbkSQiyONtzrklwO50v5dkwqoz/AHo4Zw7CTgTeMCCKatbAI8651oCXwED\nzawGMBro5ZxrC4wD7il+UTO708y6lXbTogzpBLyU1ncjyUSSx0UuB2aUcY5UXpR5nHFhPSYbMNzM\n2gP7gMbAYUXHPnbO/btoezyFYfFMoDXwetFnXR34tPhFnXPDyrjv+cC/9Ygcikjy2MzOpLAw1Pqf\nmRfV9zgUYRWGfYCGQFvn3G4z2wzULDpWvKOjo/BDX+Oca1fJ+/ZGj8hhCT2Pzex44Cmgs3Nue0Wv\nI+UW1fc4FGE9JtcBthV9gGcC8Ws+NjUz/2FdCswHNgAN/X4zq2Fmx6VyQzOrA5wBvFzp1Et5hJrH\nZtYUmAz0dc69l5Z3IGUJ/XscprAKwwlAvpmtAvoB6+OObQCuNrN1QD3gMefcj0AvYISZrQRWAKcW\nv2gZdQ09gNecc9+m8X1I6cLO42HAz4FHi7pQLU3v25EkQs1jMzvczLYCNwBDzWyrmR2a9nfl76fh\neCIiGoEiIgKoMBQRAVQYiogAKgxFRAAVhiIiQIqdrhs0aODy8vIylJTss3nzZgoKCqzsM3OH8jj3\nKY+TS6kwzMvLY+nSn053rvz8/KiTEDrlce5THienx2QREVQYiogAKgxFRAAVhiIigApDERFAhaGI\nCJClC0J9+23hrFuDBw8G4PHHH48d883kL774IgDNmjVDRKSyFBmKiJClkeEnn3wCwNixYwGoXr16\n7JjvLPrKK68A8Oc//znk1ElFLFu2DICePXsChaMCKuq1116Lbbds2RKAJk2aVDxxEhn/Pe7WrXBu\n19GjRwMwYMCA2Dnx3/9MUmQoIkKWRYZffPEFAP379484JZJus2bNAmDXrl2Vvta0adNi2+PGjQNg\n4sSJlb6uhGf79sL1u+IjQIBBgwYBcPnll8f2HXTQQaGkSZGhiAhZEhk+/PDDAEydOhWAJUuWlPma\nefPmAeDXcGnTpg0A7du3z0QSpYL27NkDwKuvvpq2a8YPvH/wwQeBoAfCwQcfnLb7SOa8/fbbAPz3\nv/9N2H/JJZcAULNmzRKvyTRFhiIiZElkeN111wGptRpNnjw54WfTpk0BeOGFF2LntG3bNl1JlAp6\n8803AViwYAEAN998c6WvuWPHjtj2mjVrAPjuu+8ARYbZLL6++O677056Tt++fQEwC3+KSUWGIiKo\nMBQRASJ+TO7SpQsQNILs3bu3zNc0aNAACB6HtmzZAsCmTZsAOPnkk2Pn7tu3L32JlXJbtWpVbLt3\n794ANG/eHIAhQ4ZU+vrxXWuk6nj33Xdj274TvnfAAYVFUefOnUNNUzxFhiIiRBAZzp07N7a9fv16\nIKgsLa0B5aqrroptd+zYEYA6deoAMGfOHADuueeeEq977LHHgJIdOyWz4vPCN2yMHz8egNq1a1f4\nur7hJP7/UBQV7VIxvrEzmXPPPTfElCSnyFBEhBAjQz8w39chARQUFCQ913eT6dWrFwC33XZb7Fit\nWrUSzvVTeD3xxBMlrnnTTTcB8MMPPwDBpA41atSo2JuQ/Zo0aRKQ2MHa1xXG1+VWlO+OER8NdujQ\nAYC6detW+vqSWfERvXfggQcCMHz48LCTU4IiQxERQowMd+/eDZQeDUIwlO75558Hgpbj/fGRoW+l\nvOGGG2LH/BAtHyH6aYKOOuqolNIu5eMn3PWfO6SnvtY/VTz33HNA0PIIMHToUEDRfjbzHe4XLlxY\n4ph/0jvhhBNCTVMyigxFRMiS4Xi+Punpp58GyhcRFuejvgkTJsT2LV68OA2pk7Ls3LkTgEWLFpU4\nNnDgwEpf/8knnwSCKd5atWoVO3bWWWdV+vqSWfubeCWbenooMhQRIYLIMNkok//85z+Vvq4fxRI/\n6qT4yBbfKu37vEl6+AH4W7duBYJpmNLlgw8+SPi9devWab2+ZFayyNC3/qfjySFdFBmKiKDCUEQE\nCPEx2a99nKmVrvwqW8uXL4/tKz7M74477sjIvX/qDjnkECDoHhE/UYMfQle/fv2Ur7tt2zYg6LLj\n/eY3v6lQOiVc8+fPB4IuUfH8cNojjzwy1DTtjyJDERFCjAynT5+e1uv5bhZr164F9j+cx3fVUcfc\nzPCrl/mhd35YHsB5550HJHaGT2b16tWxbd9g4qdnKz4ZQ7Vq+hteFfgV8HxDZrxsmJihOP2vEhEh\nSzpdV4SfJmrMmDGlnpOXlwfAs88+CwQTQEhm3H777UBiJOCfCOIn6EimYcOGsW0fCZY2dPOyyy6r\nTDIlJMXreuMn07jyyivDTk6ZFBmKiFAFI0O/VICfGHZ//LCt008/PaNpkkItW7YEElco9K37xTtO\nF+ena4vXv39/oGQneV9HKdnJd74v3ooc33Kcjind0k2RoYgIIUaG+1v0acaMGQm/X3HFFQB88skn\npV6nPNO9p7sFW1J34oknJvxMxS9/+cuk++P7Mf7qV7+qWMIkY/yUXcVbkbt37x5FcspNkaGICCoM\nRUSAEB+T/bxlftbpeL5jbvGhesmG7vnH7PKspCdVm3/MKv64pUfj7OY7W3t+0MN1110XRXLKTZGh\niAghRoY9e/YEYOTIkbF9+1sPpSz+r43vzjF27FgAGjVqVOFrSnbxjWRaG7lqmTVrVsLvTZo0AYLJ\nGbKVIkMREUKMDP0qdn7lO4CpU6cCMGrUqJSv97e//Q0I1kKW3OPXu/bU2Tq7+RUwN27cmLC/Zs2a\nQPZPlKLIUESECIbj+bWR47c7duwIBKug+Ylazz//fAD+9Kc/xV7jWxbjV0iT3ORXS/QD/IcNGxZl\ncqQMfmo1P9RuzZo1ABx99NGRpSkVigxFRMiSiRo6deqU8FMEggjj+uuvB7RGcrbzfX/99Hq+F8BJ\nJ50UWZpSochQRIQsiQxFkvF1x1K1HHHEEQCMGzcu4pSkRpGhiAgqDEVEABWGIiKACkMREUCFoYgI\noMJQRAQAS7bafaknm30BbMlccrJOM+dcw7JPyx3K49ynPE4upcJQRCRX6TFZRAQVhiIigApDEREg\nw4WhmX1TxvE8M1ud4jWfMbNeZZxjZvawmW00s3fNrGpMm1EFRZXHceeebGZ7ynu+pC7C7/GxZrbQ\nzHaZ2Y2pXL8icnWihs7A0UX/TgEeK/opOcTMqgMjgNeiTotkxA7gGuCCMG4WymOymdU2s9lmtszM\nVplZ97jDB5jZBDNbZ2aTzKxW0WvamtlcM3vHzGaZWSrL3nUH/r8rtAiom+LrJUUR5DHAIOAlYFu6\n3oeULuw8ds5tc84tAXan+70kE1ad4Q9AD+fcScCZwAMWrP/YAnjUOdcS+AoYaGY1gNFAL+dcW2Ac\ncE/xi5rZnWbWLcn9GgMfx/2+tWifZE6oeWxmjYEeFEb9Eo6wv8ehCusx2YDhZtYe2EdhwXRY0bGP\nnXP/LtoeT2FYPBNoDbxe9FlXBz4tflHnnBbFyB5h5/Eo4Gbn3D6tqxyanP4eh1UY9gEaAm2dc7vN\nbDNQs+hY8V7fjsIPfY1zrl0F7/dfoEnc70cW7ZPMCTuP84GJRV+yBkAXM9vjnJtawetJ2cLO41CF\n9ZhcB9hW9AGeCTSLO9bUzPyHdSkwH9gANPT7zayGmR2Xwv2mAf2KWpV/Dex0zpX4iyRpFWoeO+f+\nxzmX55zLAyYBA1UQZlzY3+NQhVUYTgDyzWwV0A9YH3dsA3C1ma0D6gGPOed+BHoBI8xsJbACOLX4\nRfdT1/Aq8CGwERgLDEznm5Gkws5jCV+oeWxmh5vZVuAGYKiZbTWzQ9P+rvz9NDZZREQjUEREABWG\nIiKACkMREUCFoYgIkGI/wwYNGri8vLwMJSX7bN68mYKCgp9Uj17lce5THieXUmGYl5fH0qVLK56q\nKiY/Pz/qJIROeZz7lMfJ6TFZRAQVhiIigApDERFAhaGICKDCUEQEUGEoIgKoMBQRAXJ3QSgRqUK+\n/PJLAD766KNSz2nWrHD6xIceegiA1q1bA3DMMccA0KZNm0qlQZGhiAgRR4bbthUuanbRRRcBcOqp\nhfM+XnnllUBhT/l02LlzJwBvv/02AJ06dQKgRo0aabm+iKRm+vTpALzyyisAvPXWWwC8//77pb6m\nRYsWQOHwOoBdu3YlHN+3b1+l0qTIUESECCJDXzcAcNxxhcsh+MjtsMMKF9pKd0R40kknAVBQUAAQ\nG5d59NFHp+U+Un5fffUVAH/9618BWLNmDQBvvPFG7BxF7Lnhgw8+AGDMmDEAPPnkk7Fj33//PQCp\nzLS/YcOGNKauJEWGIiKEGBn6qMzXDwJs374dgKuvvhqA0aNHp/Wed999NwCbNm0Cgr9MigjDN378\neACGDh0KlGw19BEjwM9//vPwEiYZs3XrVgBGjRpVqesce+yxQNB6nCmKDEVECDEyXLZsGRC0GsUb\nNmxY2u6zevXq2Pb9998PQI8ePQC4+OKL03YfKR8fHVx//fVA8IRQtPh7zKBBg2LbjzzyCAD169cP\nI4lSAT4fIYj8TjvtNCDorXHggQcCUKdOHQBq164de80333wDwG9/+1sgiPpOOeUUAE488cTYuQcd\ndBAABx98cJrfRSJFhiIiqDAUEQFCeEz2HatfeumlEsfGjRsHQMOGDSt9H/94fO6555Y41rNnTwAO\nOeSQSt9HUuOrKnxjWWkmTpwY254xYwYQNLb4R2j/2CXR+fbbb4HE79nKlSsBmDp1asK57dq1A2D5\n8uVAYpc534B25JFHAlCtWvRxWfQpEBHJAhmPDP/yl78AQdcK3wEa4MILL0zbfebPnw/AZ599Ftt3\n2WWXAfD73/8+bfeRsm3ZsiW2/fTTTycc84PpfQf7119/vcTrfWd5H1X26dMHgMMPPzz9iZVy+fHH\nHwG49NJLgSAaBBgyZAgA55xzTtLXJhtE0bRp0zSnsPIUGYqIEEJk6LtQ+J+NGzeOHatMHZAfzjN8\n+HAgGPIT32XD10lKuFasWBHb9p2p27dvD8DcuXMB+OGHHwB47rnnAPj73/8ee83GjRuBIMrv3r07\nENQlqstNeHwXGP898xMrxNfzDx48GIBatWqFnLr0UmQoIkIEEzX4qXsAOnbsCEDdunUBGDBgQJmv\n9522/c9FixYlHE9nPaRUTPzUSj5S952uvZo1awLwhz/8AYBJkybFjvkB/n4Qv4841JocPt9CfO+9\n9wLBBKvz5s2LneM7VVd1igxFRAghMrz22msBmDNnDgCffPJJ7JivP/IRwMsvv1zm9fy5xYdzHXXU\nUUBQtyHR+ec//1li37/+9S8ALrjggqSv8dOqJfPrX/8aSBzOJeFYsGBBwu9+mJzvH5hLFBmKiBBC\nZNi2bVsAVq1aBSS2NM6cOROAkSNHAvCLX/wCgP79+5d6vb59+wJw/PHHJ+z3Swb4CFGic8kll8S2\nfbS/ZMkSANavXw8E/x+mTJkCJE766+uQ/T4/9ZrP+1atWmUs7ZIovi4Xghb9O+64I7avW7duQOLk\nClWRIkMREVQYiogAYKmsQZCfn+/2V9Edhg8//BAIHodPOOEEAF577TUgPZM+ePn5+SxdutTKPjN3\npCOPd+zYEdv2+eSH2JXWABY/8N93oO/atSsA7733HhCsmvj4449XKn3xlMf7V3zQRDLVq1cH4Kqr\nrgKCOQk//vhjAJo3bw4Eax7F82vg+EkdMtEwU948VmQoIkLE6yZXxJ133gkEf6l840s6I0KpnPjh\nci+++CIAvXr1AkpGiNdccw0AI0aMiL3Gd8j2U6/5oXqzZs0Cgk7ZoAazTLvxxhsBeOCBB0o9Z+/e\nvUAQ0fufqfCNpx06dAASp3QLiyJDERGqSGToowuAZ599FoBDDz0U0Epq2c5P6+S7aPiJGXz3GR/p\n+2gw3q233grAunXrgKCbjn8NBP8fJDP8MDy/qqWfTm337t2xc/w6Nz5CrAg/CbT/rsevhOcn+c00\nRYYiIlSRyNB39Ix33nnnAYmTxUr28hFiaROAJuNXRfOrGvrI8M0334yd41uuNa1XZviW4pNPPhkI\nWvbjzZ49Gwiixdtvvx2AxYsXp3w/X5f8zjvvpPzaylJkKCJCFYwM/dqpvpVLcp+vr5o2bRqQ2NLo\n11hO59rbkpqzzz474Xc/5NZHhjVq1ACCZTgArrjiCgAeeughIKhLjpIiQxERVBiKiABZ/pjsh13F\nr3jnV1VTw8lPh19T96abbgIS1+f1lfW9e/cG4Jhjjgk3cVKCn8Her5rnG1b87EMA77//PhDMWF9c\n/FpJYVFkKCJCFYkM4weJd+nSJeGcr7/+GgjmvsvG9VglPfykHHfddVdsn29Iu+WWW4BgfW7fLUfC\n17JlSyDoEvX888+XOCe+exTAAQcUFkW+y1z88MywKDIUESHLI8Nk/F8QHwH4pnk/fEfDs3Jfv379\nYttPPPEEAJMnTwaCuqjiM6FLeHxUPmrUKCB4eovvSP35558DkJeXBwR56uuAo6DIUESEKhgZjh07\nFoCnnnoKgD/+8Y9AMKhfcl/8dG1vvPEGEKzn6ycWyIZOvD91vueHXyv9H//4R+zYwoULgSAS9FN4\nRUmRoYgIWR4Zjh49GoDbbrsttq99+/YADBgwAIB69eoBcOCBB4acOskGvveAXzbAD9lbu3YtoJX0\nsolf3bD4drZQZCgiQpZHhqeffjoAc+bMiTglku385LFt2rQBYOPGjYAiQyk/RYYiIqgwFBEBsvwx\nWaS8/Jo4mzZtijglUlUpMhQRQYWhiAigwlBEBADzq1GV62SzL4AtmUtO1mnmnGtY9mm5Q3mc+5TH\nyaVUGIqI5Co9JouIoMJQRATIcGFoZt+UcTzPzFaneM1nzKxXOc892cz2lPd8SV1UeWxm9cxsipm9\na2aLzax1KveQ8oswjzuY2U4zW1H0L6OLY+dsp2szqw6MAF6LOi2SEUOAFc65HmZ2LDAGOLuM10jV\nM8851zWMG4XymGxmtc1stpktM7NVZtY97vABZjbBzNaZ2SQzq1X0mrZmNtfM3jGzWWbWKMXbDgJe\nAral631I6SLI41bAHADn3Hogz8wOS987kuIi+h6HJqw6wx+AHs65k4AzgQcsWPKuBfCoc64l8BUw\n0MxqAKOBXs65tsA44J7iFzWzO82sW5L9jYEewGMZeTeSTKh5DKwEehad879AM+DINL8nSRR2HgO0\nM7OVZjbDzI5L9xuKF9ZjsgHDzaw9sA9oDPi/4h875/5dtD0euAaYCbQGXi/6rKsDnxa/qHOutDqE\nUcDNzrl98cuMSkaFncf3Av/PzFYAq4DlwN70vBUpRdh5vIzCPoLfmFkXYCpwdJreSwlhFYZ9gIZA\nW+fcbjPbDNQsOla8o6Oj8ENf45xrV8H75QMTizKgAdDFzPY456ZW8HpStlDz2Dn3FXAZQFF0sgn4\nsCLXknKLIo/99qtm9qiZNXDOFVTkemUJ6zG5DrCt6AM8k8JHGq+pmfkP61JgPrABaOj3m1mNVEJk\n59z/OOfynHN5wCRgoArCjAs1j82srpn5tR7+CLwd/+WRjAg7jw/3j+FFVSHVgO1peB9JhVUYTgDy\nzWwV0A9YH3dsA3C1ma0D6gGPOed+BHoBI8xsJbACOLX4Rcuoa5BwhZ3HLYHVZrYB6Axcm9Z3I8mE\nnce9KMzjlcDDQG+XwSFzGo4nIoJGoIiIACoMRUQAFYYiIoAKQxERQIWhiAigwlBEBFBhKCICqDAU\nEQHg/wDFEXwpmLudbQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11ce08650>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get the first images from the test-set.\n",
    "images = data.test.images[0:9]\n",
    "\n",
    "# Get the true classes for those images.\n",
    "cls_true = np.argmax(data.test.labels[0:9],1)\n",
    "\n",
    "# Plot the images and labels using our helper-function above.\n",
    "plot_images(images=images, cls_true=cls_true)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# define convolutional layer function\n",
    "def conv_fun(name,input,input_channels,output_channels):\n",
    "    # collect all nodes bellow\n",
    "    with tf.name_scope(name):\n",
    "        # w for weights\n",
    "        w = tf.Variable(tf.truncated_normal(stddev=0.5,shape=[5,5,input_channels,output_channels]),name=\"conv_w\")\n",
    "        # b for biases\n",
    "        b = tf.Variable(tf.constant(0.5,shape=[output_channels]),name=\"conv_b\")\n",
    "        # summary histogram of w & b\n",
    "        tf.summary.histogram(\"conv_w\",w)\n",
    "        tf.summary.histogram(\"conv_b\",b)\n",
    "        # convolute \n",
    "        conv_layer = tf.nn.conv2d(\n",
    "            input = input,\n",
    "            filter = w,\n",
    "            strides = [1,1,1,1],\n",
    "            padding=\"SAME\",\n",
    "            name=\"conv_op\"\n",
    "        )\n",
    "\n",
    "        # add biases\n",
    "        conv_layer = tf.add(conv_layer,b,name=\"conv_add\")\n",
    "        # relu\n",
    "        conv_layer = tf.nn.relu(conv_layer,name=\"conv_relu\")\n",
    "    \n",
    "        return conv_layer\n",
    "\n",
    "# define pooling layer function\n",
    "def pool_fun(name,input):\n",
    "    # collect all nodes bellow\n",
    "    with tf.name_scope(name):\n",
    "        # max pool\n",
    "        pool_layer = tf.nn.max_pool(\n",
    "            value = input,\n",
    "            ksize = [1,2,2,1],\n",
    "            strides = [1,2,2,1],\n",
    "            padding=\"SAME\",\n",
    "            name = \"pool_op\"\n",
    "        )\n",
    "        return pool_layer\n",
    "\n",
    "\n",
    "#define fully connected layer function     \n",
    "def fl_fun(name,input,input_channels,output_channels):\n",
    "    # collect all nodes bellow    \n",
    "    with tf.name_scope(name):\n",
    "        # weights & biases\n",
    "        w = tf.Variable(tf.truncated_normal(stddev=0.5,shape=[input_channels,output_channels]),name=\"fl_w\")\n",
    "        b = tf.Variable(tf.constant(0.5,shape=[output_channels]),name=\"fl_b\")\n",
    "        # summary histogram of w & b\n",
    "        tf.summary.histogram(\"fl_w\",w)\n",
    "        tf.summary.histogram(\"fl_b\",b)\n",
    "        # add biases\n",
    "        fl = tf.add(tf.matmul(input,w),b,name = \"fl_add\")\n",
    "        \n",
    "        return fl\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# generate different string\n",
    "def generate(learning_rate,use_two_conv,use_two_fl):\n",
    "    str_a = 2 if use_two_conv else 1\n",
    "    str_b = 2 if use_two_fl else 1\n",
    "    return \"lr_%s,conv=%s,fl=%s\" % (learning_rate,str_a,str_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# main model to evaluate different conditions \n",
    "def minst_model(lr,use_two_conv,use_two_fl):\n",
    "    # must reset\n",
    "    tf.reset_default_graph()\n",
    "    # x for input \n",
    "    x = tf.placeholder(tf.float32,[None,784],name=\"x\")\n",
    "    # reshape x to NHWC format matrix\n",
    "    x_reshape = tf.reshape(x,[-1,28,28,1],name=\"x_reshape\")\n",
    "    # summary input images\n",
    "    tf.summary.image(\"input_images\",x_reshape)\n",
    "    \n",
    "    # use two convolutional layers\n",
    "    if use_two_conv:\n",
    "        # conv1,shape :[-1,28,28,32]\n",
    "        conv1 = conv_fun(\"conv1\",x_reshape,1,32)    \n",
    "        # pool1 ,shape: [-,14,14,32]\n",
    "        pool1 = pool_fun(\"pool1\",conv1)\n",
    "        # conv2 ,shape:[-1,14,14,64]\n",
    "        conv2 = conv_fun(\"conv2\",pool1,32,64)\n",
    "        # pool2, shape:[-1,7,7,64]\n",
    "        pool2 = pool_fun(\"pool2\",conv2)\n",
    "    # only use one conv layer\n",
    "    else:\n",
    "        # conv1,shape :[-1,28,28,64]\n",
    "        conv1 = conv_fun(\"conv1\",x_reshape,1,64)    \n",
    "        # pool1 ,shape: [-,14,14,64]\n",
    "        pool1 = pool_fun(\"pool1\",conv1)\n",
    "        # pool2, shape:[-1,7,7,64]        \n",
    "        pool2 = pool_fun(\"pool2\",pool1)\n",
    "\n",
    "    # flat pool2 to shape [-1,7*7*64]\n",
    "    flat_layer = tf.reshape(pool2,[-1,7*7*64],name=\"flat_layer\")\n",
    "    \n",
    "    # use two fully connected layers\n",
    "    if use_two_fl:\n",
    "        # fl_1 (fully connected layer 1) ,shape: [-1,7*7*64,1024]\n",
    "        fl_1 = fl_fun(\"fl_1\",flat_layer,7*7*64,1024)\n",
    "        # fl_2, shape [-1,10], as logits\n",
    "        fl_2 = fl_fun(\"fl_2\",fl_1,1024,10)\n",
    "    # only use one fl\n",
    "    else:\n",
    "        # fl_2, shape [-1,10], as logits\n",
    "        fl_2 = fl_fun(\"fl_2\",flat_layer,7*7*64,10)\n",
    "        \n",
    "        \n",
    "    # predictive y class\n",
    "    y_pred_cls = tf.arg_max(fl_2,1)\n",
    "\n",
    "    \n",
    "    y_true = tf.placeholder(tf.float32,[None,10],name=\"y_true\")\n",
    "    y_true_cls = tf.arg_max(y_true,1,name=\"y_true_cls\")\n",
    "\n",
    "    # cost \n",
    "    with tf.name_scope(\"cost\"):\n",
    "        cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_true,logits=fl_2,name=\"cross_entropy\")\n",
    "        cost = tf.reduce_mean(cross_entropy)\n",
    "        # summary cost \n",
    "        tf.summary.scalar(\"cost\",cost)\n",
    "    # use  AdamOptimizer to train the model    \n",
    "    with tf.name_scope(\"train\"):\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(cost)\n",
    "    # accuracy    \n",
    "    with tf.name_scope(\"accuracy\"):\n",
    "        equals = tf.equal(y_pred_cls,y_true_cls)\n",
    "        accuracy = tf.reduce_mean(tf.cast(equals,dtype=tf.float32))\n",
    "        # summary accuracy\n",
    "        tf.summary.scalar(\"accuracy\",accuracy)\n",
    "        \n",
    "    # initial session and variables\n",
    "    session = tf.Session()\n",
    "    session.run(tf.global_variables_initializer())\n",
    "    # merge all summaries\n",
    "    summary_merged = tf.summary.merge_all()\n",
    "    # write graph and summaries to train writer\n",
    "    train_writer = tf.summary.FileWriter('./log/train_'+generate(lr,use_two_conv,use_two_fl)    ,graph=session.graph)\n",
    "    # writw summaries to test writer\n",
    "    test_writer = tf.summary.FileWriter('./log/test_'+generate(lr,use_two_conv,use_two_fl) )\n",
    "    \n",
    "    batch_size = 100;\n",
    "    with tf.name_scope(\"train_iterals\"):\n",
    "        for i in range(6000):\n",
    "            batch = data.train.next_batch(batch_size)\n",
    "            feed_dict_batch = {x:batch[0],y_true:batch[1]}\n",
    "            print i\n",
    "            if i % 5 == 0:\n",
    "                s,acc =  session.run([summary_merged,accuracy],feed_dict=feed_dict_batch)\n",
    "                train_writer.add_summary(s,i)\n",
    "                print 'Train_'+generate(lr,use_two_conv,use_two_fl)+\"acc:{:0.1%}\".format(acc)\n",
    "            if i % 10 == 0:\n",
    "                feed_dict_test = {\n",
    "                    x : data.test.images,\n",
    "                    y_true: data.test.labels\n",
    "                }\n",
    "                s,acc = session.run([summary_merged,accuracy],feed_dict=feed_dict_test)\n",
    "                test_writer.add_summary(s,i)\n",
    "                print 'Test_'+generate(lr,use_two_conv,use_two_fl)+\" acc:{:0.1%}\".format(acc)\n",
    "            \n",
    "            \n",
    "            session.run(optimizer,feed_dict=feed_dict_batch)    \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# different conditions\n",
    "# learning rate in [0.001,0.0001]\n",
    "for lr in [0.001,0.0001]:\n",
    "    # use one or two conv layers\n",
    "    for use_two_conv in [True,False]:\n",
    "        # use one or two fl\n",
    "        for use_two_fl in [True,False]:\n",
    "            # start model , and evaluate it\n",
    "            minst_model(lr,use_two_conv,use_two_fl)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
